package com.chance.cc.crawler.development.bootstrap.autohome.forum;

import com.alibaba.fastjson.JSON;
import com.chance.cc.crawler.core.CrawlerEnum;
import com.chance.cc.crawler.core.downloader.HttpConfig;
import com.chance.cc.crawler.core.downloader.proxy.Proxy;
import com.chance.cc.crawler.core.filter.FilterUtils;
import com.chance.cc.crawler.core.record.CrawlerRecord;
import com.chance.cc.crawler.core.record.CrawlerRequestRecord;
import com.chance.cc.crawler.development.controller.DevCrawlerController;
import org.apache.commons.lang3.StringUtils;


import java.util.HashMap;
import java.util.Map;

import static com.chance.cc.crawler.core.CrawlerEnum.CrawlerRequestType.filter;


/**
 * @Author: ZhaoHhuan on 2021/11/1 12:28
 * @Email: 18638575967@163.com
 * @Description:
 *      汽车之家论坛单个帖子采集
**/
public class ForumArticle {
    private static String domain = "autohome";
    private static String site = "forum";
    private static final String OrderType = "order_type";//回复排序 1  发布排序 2

    private static Proxy proxy = new Proxy();
    static {
        //代理配置
        proxy.setHost("http-dyn.abuyun.com");
        proxy.setPort(9020);
        proxy.setUsername("HL89Q19E86E2987D");
        proxy.setPassword("71F33D94CE5F7BF2");
    }

    public static void main(String[] args) {
        forumArticleTest();

    }

    public static void forumArticleTest(){
        String url = "http://club.autohome.com.cn/bbs/thread/ff1a385d089fb9ca/99439785-1.html";
        HttpConfig httpConfig = HttpConfig.me(domain);
        httpConfig.setHttpSite(site);

        CrawlerRequestRecord requestRecord = CrawlerRequestRecord.builder()
                .startPageRequest(domain, CrawlerEnum.CrawlerRequestType.turnPage)
                .httpUrl(url)
                .releaseTime(System.currentTimeMillis())
                .httpConfig(httpConfig)
                .domain(domain)
                .filter(CrawlerEnum.CrawlerRecordFilter.dateRange)
                .addFilterInfo(FilterUtils.memoryFilterKeyInfo(domain))
                .addFilterInfo(FilterUtils.dateRangeFilterInfo(24*30*2,null))
                .needParsed(false)
                .proxy(proxy)
                .build();
        requestRecord.setDownload(false);
        requestRecord.setSkipPipeline(true);

        requestRecord.tagsCreator().bizTags().addDomain(domain);
        requestRecord.tagsCreator().bizTags().addSite(site);
        requestRecord.tagsCreator().bizTags().addSiteBiz("forum_simple");
        requestRecord.tagsCreator().bizTags().addCustomKV(OrderType,"1");
        Map<String,Object> extras = new HashMap<>();
        extras.put("path","F:\\微信\\WeChat Files\\wxid_l2bvaq20ulr822\\FileStorage\\File\\2021-11\\汽车之家论坛Redis-8-3_2.log");
        requestRecord.getHttpRequest().setExtras(extras);

        DevCrawlerController crawlerJob = DevCrawlerController.builder()
                .triggerInfo(domain, domain, System.currentTimeMillis(), domain)
                .crawlerRequestQueue(DevCrawlerController.devRequestQueue(domain)) //内存队列
                .consoleResultPipeline() //控制台输t出
//                .fileResultPipeline("D:\\chance\\log\\test.log",true)
                .requestRecord(requestRecord)
                .build("com/chance/cc/crawler/development/scripts/autohome");
        //添加评论去重信息
        CrawlerRecord filterCrawlerRecord = new CrawlerRecord();
        filterCrawlerRecord.setFilter(CrawlerEnum.CrawlerRecordFilter.dateRange);
        filterCrawlerRecord.addFilterInfo(FilterUtils.memoryFilterKeyInfo(StringUtils.joinWith("-",filter,domain,"comment")));
        filterCrawlerRecord.addFilterInfo(FilterUtils.dateRangeFilterInfo(6,null));
        requestRecord.tagsCreator().bizTags().addCustomKV("comment_record_filter_info", JSON.toJSONString(filterCrawlerRecord));
        crawlerJob.getCrawlerJob().getScheduleTags().getCategoryTag().addLabelTag(CrawlerEnum.CrawlerDataType.comment.enumVal());
        crawlerJob.start();
    }
}
